{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import qlib\n",
    "import pandas as pd\n",
    "from qlib.utils.time import Freq\n",
    "from qlib.utils import flatten_dict\n",
    "from qlib.contrib.evaluate import backtest_daily\n",
    "from qlib.contrib.evaluate import risk_analysis\n",
    "from qlib.contrib.strategy import TopkDropoutStrategy\n",
    "from qlib.utils import init_instance_by_config\n",
    "\n",
    "from qlib.workflow import R\n",
    "from qlib.workflow.record_temp import SignalRecord, PortAnaRecord, SigAnaRecord\n",
    "\n",
    "from qlib.contrib.report import analysis_model, analysis_position\n",
    "\n",
    "from qlib.backtest import backtest, executor,exchange\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[23464:MainThread](2024-04-08 21:27:28,940) INFO - qlib.Initialization - [config.py:416] - default_conf: client.\n",
      "[23464:MainThread](2024-04-08 21:27:28,952) INFO - qlib.Initialization - [__init__.py:74] - qlib successfully initialized based on client settings.\n",
      "[23464:MainThread](2024-04-08 21:27:28,954) INFO - qlib.Initialization - [__init__.py:76] - data_path={'__DEFAULT_FREQ': WindowsPath('C:/Users/Lincoln/.qlib/qlib_data/ch_data')}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "qlib.init(provider_uri='~/.qlib/qlib_data/ch_data')\n",
    "\n",
    "# MARKET = \"all\"\n",
    "# BENCHMARK = \"000300\"\n",
    "# EXP_NAME = \"tutorial_exp\"\n",
    "# CSI300_BENCH = \"000300\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'class': 'Alpha158',\n",
      " 'kwargs': {'end_time': '2023-12-29',\n",
      "            'instruments': 'all',\n",
      "            'start_time': '2021-02-01'},\n",
      " 'module_path': 'qlib.contrib.data.handler'}\n"
     ]
    }
   ],
   "source": [
    "# 数据参数\n",
    "handler_kwargs = {\n",
    "        \"start_time\": \"2021-02-01\",\n",
    "        \"end_time\": \"2023-12-29\",\n",
    "        # \"fit_start_time\": \"2020-01-01\",\n",
    "        # \"fit_end_time\": \"2022-05-31\",\n",
    "        \"instruments\": 'all',\n",
    "}\n",
    "\n",
    "# 因子生成参数\n",
    "handler_conf = {\n",
    "    \"class\": \"Alpha158\",\n",
    "    \"module_path\": \"qlib.contrib.data.handler\",\n",
    "    \"kwargs\": handler_kwargs,\n",
    "}\n",
    "\n",
    "pprint(handler_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[23464:MainThread](2024-04-08 21:53:26,125) INFO - qlib.timer - [log.py:128] - Time cost: 72.882s | Loading data Done\n",
      "[23464:MainThread](2024-04-08 21:53:26,680) INFO - qlib.timer - [log.py:128] - Time cost: 0.170s | DropnaLabel Done\n",
      "[23464:MainThread](2024-04-08 21:53:28,389) INFO - qlib.timer - [log.py:128] - Time cost: 1.707s | CSZScoreNorm Done\n",
      "[23464:MainThread](2024-04-08 21:53:28,405) INFO - qlib.timer - [log.py:128] - Time cost: 2.277s | fit & process data Done\n",
      "[23464:MainThread](2024-04-08 21:53:28,407) INFO - qlib.timer - [log.py:128] - Time cost: 75.163s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>KMID</th>\n",
       "      <th>KLEN</th>\n",
       "      <th>KMID2</th>\n",
       "      <th>KUP</th>\n",
       "      <th>KUP2</th>\n",
       "      <th>KLOW</th>\n",
       "      <th>KLOW2</th>\n",
       "      <th>KSFT</th>\n",
       "      <th>KSFT2</th>\n",
       "      <th>OPEN0</th>\n",
       "      <th>...</th>\n",
       "      <th>VSUMN10</th>\n",
       "      <th>VSUMN20</th>\n",
       "      <th>VSUMN30</th>\n",
       "      <th>VSUMN60</th>\n",
       "      <th>VSUMD5</th>\n",
       "      <th>VSUMD10</th>\n",
       "      <th>VSUMD20</th>\n",
       "      <th>VSUMD30</th>\n",
       "      <th>VSUMD60</th>\n",
       "      <th>LABEL0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2021-02-01</th>\n",
       "      <th>000001.SZ</th>\n",
       "      <td>0.067392</td>\n",
       "      <td>0.099562</td>\n",
       "      <td>0.676886</td>\n",
       "      <td>0.019128</td>\n",
       "      <td>0.192124</td>\n",
       "      <td>0.013042</td>\n",
       "      <td>0.130990</td>\n",
       "      <td>0.061306</td>\n",
       "      <td>0.615752</td>\n",
       "      <td>0.936863</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.071738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000002.SZ</th>\n",
       "      <td>0.003575</td>\n",
       "      <td>0.030747</td>\n",
       "      <td>0.116282</td>\n",
       "      <td>0.001790</td>\n",
       "      <td>0.058210</td>\n",
       "      <td>0.025382</td>\n",
       "      <td>0.825508</td>\n",
       "      <td>0.027167</td>\n",
       "      <td>0.883580</td>\n",
       "      <td>0.996437</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.014819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000063.SZ</th>\n",
       "      <td>0.010632</td>\n",
       "      <td>0.027830</td>\n",
       "      <td>0.382030</td>\n",
       "      <td>0.008754</td>\n",
       "      <td>0.314550</td>\n",
       "      <td>0.008444</td>\n",
       "      <td>0.303420</td>\n",
       "      <td>0.010322</td>\n",
       "      <td>0.370899</td>\n",
       "      <td>0.989480</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.006936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000069.SZ</th>\n",
       "      <td>0.013679</td>\n",
       "      <td>0.022793</td>\n",
       "      <td>0.600141</td>\n",
       "      <td>0.004549</td>\n",
       "      <td>0.199574</td>\n",
       "      <td>0.004565</td>\n",
       "      <td>0.200285</td>\n",
       "      <td>0.013695</td>\n",
       "      <td>0.600852</td>\n",
       "      <td>0.986506</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.038808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000100.SZ</th>\n",
       "      <td>0.037933</td>\n",
       "      <td>0.067813</td>\n",
       "      <td>0.559382</td>\n",
       "      <td>0.018390</td>\n",
       "      <td>0.271180</td>\n",
       "      <td>0.011490</td>\n",
       "      <td>0.169438</td>\n",
       "      <td>0.031034</td>\n",
       "      <td>0.457641</td>\n",
       "      <td>0.963453</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.025612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 159 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           KMID      KLEN     KMID2       KUP      KUP2  \\\n",
       "datetime   instrument                                                     \n",
       "2021-02-01 000001.SZ   0.067392  0.099562  0.676886  0.019128  0.192124   \n",
       "           000002.SZ   0.003575  0.030747  0.116282  0.001790  0.058210   \n",
       "           000063.SZ   0.010632  0.027830  0.382030  0.008754  0.314550   \n",
       "           000069.SZ   0.013679  0.022793  0.600141  0.004549  0.199574   \n",
       "           000100.SZ   0.037933  0.067813  0.559382  0.018390  0.271180   \n",
       "\n",
       "                           KLOW     KLOW2      KSFT     KSFT2     OPEN0  ...  \\\n",
       "datetime   instrument                                                    ...   \n",
       "2021-02-01 000001.SZ   0.013042  0.130990  0.061306  0.615752  0.936863  ...   \n",
       "           000002.SZ   0.025382  0.825508  0.027167  0.883580  0.996437  ...   \n",
       "           000063.SZ   0.008444  0.303420  0.010322  0.370899  0.989480  ...   \n",
       "           000069.SZ   0.004565  0.200285  0.013695  0.600852  0.986506  ...   \n",
       "           000100.SZ   0.011490  0.169438  0.031034  0.457641  0.963453  ...   \n",
       "\n",
       "                       VSUMN10  VSUMN20  VSUMN30  VSUMN60  VSUMD5  VSUMD10  \\\n",
       "datetime   instrument                                                        \n",
       "2021-02-01 000001.SZ       NaN      NaN      NaN      NaN     NaN      NaN   \n",
       "           000002.SZ       NaN      NaN      NaN      NaN     NaN      NaN   \n",
       "           000063.SZ       NaN      NaN      NaN      NaN     NaN      NaN   \n",
       "           000069.SZ       NaN      NaN      NaN      NaN     NaN      NaN   \n",
       "           000100.SZ       NaN      NaN      NaN      NaN     NaN      NaN   \n",
       "\n",
       "                       VSUMD20  VSUMD30  VSUMD60    LABEL0  \n",
       "datetime   instrument                                       \n",
       "2021-02-01 000001.SZ       NaN      NaN      NaN  0.071738  \n",
       "           000002.SZ       NaN      NaN      NaN -0.014819  \n",
       "           000063.SZ       NaN      NaN      NaN  0.006936  \n",
       "           000069.SZ       NaN      NaN      NaN  0.038808  \n",
       "           000100.SZ       NaN      NaN      NaN -0.025612  \n",
       "\n",
       "[5 rows x 159 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hd = init_instance_by_config(handler_conf)\n",
    "df = hd.fetch()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def factors_process(df):\n",
    "    # 剔除全部为NA的因子\n",
    "    df = df.dropna(axis=1, how='all')\n",
    "\n",
    "    # 剔除行值全相同的列\n",
    "    cols_to_drop = []\n",
    "    for col in df.columns:\n",
    "        if df[col].nunique() == 1:\n",
    "            cols_to_drop.append(col)\n",
    "    df = df.drop(cols_to_drop, axis=1)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_p = factors_process(df)\n",
    "outpath = '../量化data/alpha158/'\n",
    "columns = df_p.columns\n",
    "for i in columns:\n",
    "    df_A = df_p[i].unstack()\n",
    "    df_A = df_A.apply(lambda x: x.fillna(method='ffill'))  # 填补因子缺失值\n",
    "    df_A.to_csv(outpath + i + '.csv')\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13 (main, Aug 25 2022, 23:51:50) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f52f256adf2cc158a51e888fcb9b6688be269c3160efd3d6e246b7ec509c991c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
